﻿using HtmlAgilityPack;
using SqlSugar;

namespace MyCrawlerDemo
{
    public class CrawlerService
    {
        /// <summary>
        /// 连接配置
        /// </summary>
        public ConnectionConfig? Config { get; set; }

        public CrawlerService(ConnectionConfig config) 
        { 
            Config = config;
        }
        /// <summary>
        /// 数据库初始化
        /// </summary>
        public bool Init()
        {
            SnowFlakeSingle.WorkId = 2025;
            using SqlSugarClient db = new(Config);
            var categoryExists = db.DbMaintenance.GetTableInfoList().FirstOrDefault(l => l.Name == "Category");
            var commodityExists = db.DbMaintenance.GetTableInfoList().FirstOrDefault(l => l.Name == "Commodity");
            if (categoryExists == null)
            {
                db.CodeFirst.InitTables<JdCategory>();
                db.Deleteable<JdCategory>().ExecuteCommand();
            }
            if (commodityExists == null)
            {
                db.CodeFirst.InitTables<JdCommodity>();
                db.Deleteable<JdCommodity>().ExecuteCommand();
            }
            return true;
        }
        /// <summary>
        /// 保存目录
        /// </summary>
        /// <param name="category"></param>
        /// <returns></returns>
        public Tuple<bool, string?> SaveCategory(JdCategory category)
        {
            using SqlSugarClient db = new(Config);
            var exists = db.Queryable<JdCategory>().Where(l => l.Name == category.Name).Any();
            if (exists)
            {
                return new Tuple<bool, string?>(false, "目录已存在");
            }
            else
            {
                db.Insertable(category).ExecuteCommand();
                return new Tuple<bool, string?>(true, "保存成功");
            }
        }
        /// <summary>
        /// 批量保存目录
        /// </summary>
        /// <param name="categories"></param>
        /// <returns></returns>
        public Tuple<bool, string?> SaveCategories(List<JdCategory> categories)
        {
            using SqlSugarClient db = new SqlSugarClient(Config);
            var result = db.InsertNav(categories).Include(l => l.Children).ThenInclude(l => l.Children).ExecuteCommand(); 
            string msg = result ? "保存成功" : "保存失败";
            return new Tuple<bool, string?>(true, msg);
        }
        /// <summary>
        /// 获取所有目录
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public List<JdCategory> GetCategories(string url)
        {
            List<JdCategory> list = new();
            string html = HttpClientHelper.Post(url);
            HtmlDocument documet = new();
            documet.LoadHtml(html);
            var nodes = documet.DocumentNode.SelectNodes("//*[@class='category-item m']");
            int a = 0;
            int b = 0;
            int c = 0;
            // 获取第一级目录
            foreach (var node in nodes)
            {
                var titlenode = node.SelectSingleNode("./div[@class='mt']/h2/span");
                JdCategory first = new()
                {
                    Code = $"C{a}", 
                    Name = titlenode.InnerText,
                    Level = 1
                };
                var contentnode = node.SelectNodes("./div[@class='mc']/div[@class='items']/dl[@class='clearfix']");
                if (contentnode != null && contentnode.Count > 0)
                {
                    first.Children = new();
                    foreach (var content in contentnode)
                    {
                        var secondnode = content.SelectSingleNode("./dt/a");
                        JdCategory second = new()
                        {
                            Code = $"C{a}{b}",
                            ParentCode = first.Code,
                            Name = secondnode.InnerText,
                            Level = 2,
                            Url = secondnode.Attributes["href"].Value
                        };
                        var thirdnodes = content.SelectNodes("./dd/a");
                        if (thirdnodes != null && thirdnodes.Count > 0)
                        {
                            second.Children = new();
                            foreach (var thirdnode in thirdnodes)
                            {
                                JdCategory third = new()
                                {
                                    Code = $"C{a}{b}{c}",
                                    ParentCode = second.Code,
                                    Name = thirdnode.InnerText,
                                    Level = 3,
                                    Url = thirdnode.Attributes["href"].Value
                                };
                                second.Children.Add(third);
                                c++;
                            }
                        }
                        first.Children.Add(second);
                        b++;
                    }
                }
                list.Add(first);
                a++;
            }
            return list;
        }
        /// <summary>
        /// 获取并保存所有目录
        /// </summary>
        /// <param name="url">获取目录的地址</param>
        /// <returns></returns>
        public bool GetCategoriesAndSave(string url)
        {
            var lists = GetCategories(url);
            return SaveCategories(lists).Item1;
        }
        /// <summary>
        /// 获取所有商品并保存
        /// </summary>
        /// <param name="cookie">进入商品页面需要的cookie</param>
        /// <returns>成功保存数量</returns>
        public int GetCommoditiesAndSave(string cookie)
        {
            int result = 0;
            using SqlSugarClient client = new(Config);
            int pagenumber = 1;
            int pageSize = 100;
            int total = 0;
            List<JdCategory>? categories;
            List<JdCommodity> commodities = new();
            ParallelOptions parallelOptions = new();
            parallelOptions.MaxDegreeOfParallelism = 10;
            do
            {
                categories = client.Queryable<JdCategory>().Where(l => !string.IsNullOrEmpty(l.Url) && l.Url.StartsWith("//list.jd.com")).ToPageList(pagenumber, pageSize, ref total);
                if (categories != null && categories.Count > 0)
                {
                    Parallel.ForEach(categories, parallelOptions, category =>
                    {
                        for (int i = 1; i <= 200; i++)
                        {

                            string response = HttpClientHelper.PostUseHandleCookie($"https:{category.Url}&stock=1&isList=1&page={i}&s={30 * (i - 1) + 1}&click=0", cookie);
                            HtmlDocument document = new();
                            document.LoadHtml(response);
                            var goodnodes = document.DocumentNode.SelectNodes("//div[@id='J_searchWrap']/div[@id='J_main']/div[@class='m-list']/div[@class='ml-wrap']/div[@id='J_goodsList']/ul[@class='gl-warp clearfix']/li");
                            if (goodnodes != null && goodnodes.Count > 0)
                            {
                                foreach (var goodnode in goodnodes)
                                {
                                    var categoryId = category.Id;
                                    var pricenode = goodnode.SelectSingleNode("./div/div[@class='p-price']/strong/i");
                                    var urlnode = goodnode.SelectSingleNode("./div/div[@class='p-name']/a");
                                    var namenode = urlnode.SelectSingleNode("./em");
                                    commodities.Add(new JdCommodity()
                                    {
                                        Name = namenode?.InnerText,
                                        Url = urlnode?.Attributes["href"].ToString(),
                                        Price = decimal.Parse(pricenode?.InnerText ?? "0"),
                                        CreateDateTime = DateTime.Now
                                    });
                                }
                                result += client.Insertable<JdCommodity>(commodities).ExecuteReturnSnowflakeIdList().Count;
                                commodities.Clear();
                            }
                        }
                    });
                }
            }
            while (pagenumber++ * pageSize <= total);
            return result;
        }
    }
}

